import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.PrintWriter;
import java.util.ArrayDeque;
import java.util.Calendar;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Queue;
import java.util.Set;

import org.htmlparser.Node;
import org.htmlparser.Parser;
import org.htmlparser.tags.ImageTag;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeIterator;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;


public class getphotos {

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		// TODO Auto-generated method stub
		 String beginUrl = "http://www.169pp.com/info/4_1.htm";
         getphotos crawler = new getphotos();
         crawler.crawl(beginUrl);
	}	
	
	private Set<String> alreadyCrawledSet;
	private Queue<String> unCrawlQueue;
	private static Set<String> picSrcSet;
	private static final int MAX_COUNT = 10;
	private String newFolder;
	public getphotos() {
		//super();
		this.alreadyCrawledSet = new HashSet();
		this.unCrawlQueue = new ArrayDeque<String>();
		this.picSrcSet = new HashSet<String>();
		Calendar cal = Calendar.getInstance();
		int month = cal.get(Calendar.MONTH)+1;
		int day = cal.get(Calendar.DAY_OF_MONTH);
		int hour = cal.get(Calendar.HOUR_OF_DAY);
        int minute = cal.get(Calendar.MINUTE);
        int second = cal.get(Calendar.SECOND);
         
        String folder = month+"_"+day+"_"+hour+"_"+minute+"_"+second;  
        newFolder = "C:\\"+folder+"\\";
        File dir = new File(newFolder);
        if( null == dir || !dir.exists())
        {
            dir.mkdir();
        }
	}
	
	public void crawl(String beginUrl){
		Parser parser = null;
		
		try {
			parser = new Parser(beginUrl);
			parser.setEncoding(parser.getEncoding());
			NodeList nodeList = parser.parse(null);
			parseNodeList(nodeList);
		} catch (ParserException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		alreadyCrawledSet.add(beginUrl);
		while(!unCrawlQueue.isEmpty() && alreadyCrawledSet.size()< MAX_COUNT)
        {
            String newUrl = unCrawlQueue.remove();
            try{
                    parser.setResource(newUrl);
                    parser.setEncoding(parser.getEncoding());
                    NodeList nl = parser.parse(null);
                    parseNodeList(nl);
            } catch (ParserException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            alreadyCrawledSet.add(newUrl);
        }
        System.out.println("Crawl Finish!");
         
        FileOutputStream fos = null;
        PrintWriter pw = null;
        //把得到的全部图片地址写到文件中
        try {
            String picFile = newFolder+"picAddrs.txt";
            fos = new FileOutputStream(new File(picFile));
            pw = new PrintWriter(fos);
            Iterator<String> iter = picSrcSet.iterator();
            while(iter.hasNext())
            {
                String str = iter.next().toString()+("\r\n");
                pw.write(str);
                pw.flush();
            }
             
        } catch (FileNotFoundException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }finally
        {
            if(null!=pw)
            {
                pw.close();
                pw = null;
            }
        }
        System.out.println("====== All the pic address have been writen to the file!");
		
	}

	private void parseNodeList(NodeList nodeList) {
		NodeIterator iter = nodeList.elements();
        try {
            while(iter.hasMoreNodes())
            {
                Node node = iter.nextNode();
                if(node.getClass() == LinkTag.class)
                {
                    LinkTag tag = (LinkTag)node;
                    String href = tag.getLink();
                    //System.out.println("===find a link: "+href);
                    if(null != href)
                    {
                        if(!alreadyCrawledSet.contains(href) && (href.indexOf("www.169pp.com")!=-1) && href.endsWith("htm"))
                        {
                            System.out.println("----find link: "+href);
                            unCrawlQueue.add(href);
                        }
                    }
                }
                else
                {
                    NodeList childList = node.getChildren();
                    if( childList == null)
                    {
                        //说明是末尾节点，打印
                        //System.out.println("class: "+node.getClass()+",text: "+node.getText());
                        if(node instanceof ImageTag)
                        {
                            ImageTag tag = (ImageTag)node;
                            String picURL = tag.getImageURL();
                            if(picURL.endsWith(".jpg")||picURL.endsWith(".jpeg"))//||picURL.endsWith(".gif"))
                            {
                                picSrcSet.add(picURL);
                            }
                        }
                    }
                    else
                    {
                        parseNodeList(childList);
                    }
                     
                }
                 
            }
        } catch (ParserException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
		
	}
	
	public void displayAlreadyCrawlUrl()
    {
        System.out.println("=================================");
        Iterator<String> iter = alreadyCrawledSet.iterator();
        while(iter.hasNext())
        {
            System.out.println(iter.next());
        }
         
        System.out.println("=================================");
        Iterator<String> picIter = picSrcSet.iterator();
        while(picIter.hasNext())
        {
            System.out.println(picIter.next());
        }
         
    }
	
	

}
